{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature: Out-Of-Fold Predictions from a Siamese LSTM with Attention"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"assets/siamese-lstm-attention.png\" alt=\"Network Architecture\" style=\"height: 700px;\" />"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This utility package imports `numpy`, `pandas`, `matplotlib` and a helper `kg` module into the root namespace."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pygoose import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import gc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "kg.gpu.cuda_use_gpus(gpu_ids=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras import backend as K\n",
    "from keras.models import Model, Sequential\n",
    "from keras.layers import *\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Config"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Automatically discover the paths to various data folders and compose the project structure."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "project = kg.Project.discover()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Identifier for storing these features on disk and referring to them later."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "feature_list_id = 'oofp_nn_siamese_lstm_attention'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Make subsequent NN runs reproducible."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "RANDOM_SEED = 42"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.random.seed(RANDOM_SEED)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Word embedding lookup matrix."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "embedding_matrix = kg.io.load(project.aux_dir + 'fasttext_vocab_embedding_matrix.pickle')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Padded sequences of word indices for every question."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train_q1 = kg.io.load(project.preprocessed_data_dir + 'sequences_q1_fasttext_train.pickle')\n",
    "X_train_q2 = kg.io.load(project.preprocessed_data_dir + 'sequences_q2_fasttext_train.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_test_q1 = kg.io.load(project.preprocessed_data_dir + 'sequences_q1_fasttext_test.pickle')\n",
    "X_test_q2 = kg.io.load(project.preprocessed_data_dir + 'sequences_q2_fasttext_test.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_train = kg.io.load(project.features_dir + 'y_train.pickle')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Word embedding properties."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "EMBEDDING_DIM = embedding_matrix.shape[-1]\n",
    "VOCAB_LENGTH = embedding_matrix.shape[0]\n",
    "MAX_SEQUENCE_LENGTH = X_train_q1.shape[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "300 101442 30\n"
     ]
    }
   ],
   "source": [
    "print(EMBEDDING_DIM, VOCAB_LENGTH, MAX_SEQUENCE_LENGTH)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def contrastive_loss(y_true, y_pred):\n",
    "    \"\"\"\n",
    "    Contrastive loss from Hadsell-et-al.'06\n",
    "    http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf\n",
    "    \"\"\"    \n",
    "    margin = 1\n",
    "    return K.mean((1 - y_true) * K.square(y_pred) +\n",
    "                   y_true * K.square(K.maximum(margin - y_pred, 0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class AttentionWithContext(Layer):\n",
    "    \"\"\"\n",
    "    Attention operation, with a context/query vector, for temporal data.\n",
    "    Supports Masking.\n",
    "    \n",
    "    Follows the work of Yang et al. [https://www.cs.cmu.edu/~diyiy/docs/naacl16.pdf]\n",
    "    \"Hierarchical Attention Networks for Document Classification\" by using a context\n",
    "    vector to assist the attention.\n",
    "    \n",
    "    # Input shape\n",
    "        3D tensor with shape: `(samples, steps, features)`.\n",
    "    # Output shape\n",
    "        2D tensor with shape: `(samples, features)`.\n",
    "\n",
    "    Just put it on top of an RNN Layer (GRU/LSTM/SimpleRNN) with return_sequences=True.\n",
    "    \n",
    "    The dimensions are inferred based on the output shape of the RNN.\n",
    "    Example:\n",
    "        model.add(LSTM(64, return_sequences=True))\n",
    "        model.add(AttentionWithContext())\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, init='glorot_uniform',\n",
    "                 kernel_regularizer=None, bias_regularizer=None,\n",
    "                 kernel_constraint=None, bias_constraint=None,  **kwargs):\n",
    "        \n",
    "        self.supports_masking = True\n",
    "        self.init = initializers.get(init)\n",
    "        self.kernel_initializer = initializers.get('glorot_uniform')\n",
    "\n",
    "        self.kernel_regularizer = regularizers.get(kernel_regularizer)\n",
    "        self.bias_regularizer = regularizers.get(bias_regularizer)\n",
    "\n",
    "        self.kernel_constraint = constraints.get(kernel_constraint)\n",
    "        self.bias_constraint = constraints.get(bias_constraint)\n",
    "\n",
    "        super(AttentionWithContext, self).__init__(**kwargs)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        self.kernel = self.add_weight(\n",
    "            (input_shape[-1], 1),\n",
    "            initializer=self.kernel_initializer,\n",
    "            name='{}_W'.format(self.name),\n",
    "            regularizer=self.kernel_regularizer,\n",
    "            constraint=self.kernel_constraint\n",
    "        )\n",
    "        self.b = self.add_weight(\n",
    "            (input_shape[1],),\n",
    "            initializer='zero',\n",
    "            name='{}_b'.format(self.name),\n",
    "            regularizer=self.bias_regularizer,\n",
    "            constraint=self.bias_constraint\n",
    "        )\n",
    "        self.u = self.add_weight(\n",
    "            (input_shape[1],),\n",
    "            initializer=self.kernel_initializer,\n",
    "            name='{}_u'.format(self.name),\n",
    "            regularizer=self.kernel_regularizer,\n",
    "            constraint=self.kernel_constraint\n",
    "        )\n",
    "        self.built = True\n",
    "\n",
    "    def compute_mask(self, input, mask):\n",
    "        return None\n",
    "\n",
    "    def call(self, x, mask=None):\n",
    "        multdata = K.dot(x, self.kernel)     # (x, 40, 300) * (300, 1) => (x, 40, 1)\n",
    "        multdata = K.squeeze(multdata, -1)   # (x, 40)\n",
    "        multdata = multdata + self.b         # (x, 40) + (40,)\n",
    "\n",
    "        multdata = K.tanh(multdata)          # (x, 40)\n",
    "\n",
    "        multdata = multdata * self.u         # (x, 40) * (40, 1) => (x, 1)\n",
    "        multdata = K.exp(multdata)           # (x, 1)\n",
    "\n",
    "        # Apply mask after the exp. will be re-normalized next.\n",
    "        if mask is not None:\n",
    "            mask = K.cast(mask, K.floatx())  # (x, 40)\n",
    "            multdata = mask * multdata       # (x, 40) * (x, 40, )\n",
    "\n",
    "        # In some cases, especially in the early stages of training, the sum may be almost zero\n",
    "        # and this results in NaN's. A workaround is to add a very small positive number ε to the sum.\n",
    "        # a /= K.cast(K.sum(a, axis=1, keepdims=True), K.floatx())\n",
    "        multdata /= K.cast(K.sum(multdata, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n",
    "        multdata = K.expand_dims(multdata)\n",
    "        weighted_input = x * multdata\n",
    "        return K.sum(weighted_input, axis=1)\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        return (input_shape[0], input_shape[-1],)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_model(params):\n",
    "    embedding_layer = Embedding(\n",
    "        VOCAB_LENGTH,\n",
    "        EMBEDDING_DIM,\n",
    "        weights=[embedding_matrix],\n",
    "        input_length=MAX_SEQUENCE_LENGTH,\n",
    "        trainable=False,\n",
    "    )\n",
    "    lstm_layer = LSTM(\n",
    "        params['num_lstm'],\n",
    "        dropout=params['lstm_dropout_rate'],\n",
    "        recurrent_dropout=params['lstm_dropout_rate'],\n",
    "        return_sequences=True,\n",
    "    )\n",
    "    attention_layer = AttentionWithContext()\n",
    "\n",
    "    sequence_1_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
    "    embedded_sequences_1 = embedding_layer(sequence_1_input)\n",
    "    x1 = attention_layer(lstm_layer(embedded_sequences_1))\n",
    "\n",
    "    sequence_2_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
    "    embedded_sequences_2 = embedding_layer(sequence_2_input)\n",
    "    y1 = attention_layer(lstm_layer(embedded_sequences_2))\n",
    "\n",
    "    merged = concatenate([x1, y1])\n",
    "    merged = Dropout(params['dense_dropout_rate'])(merged)\n",
    "    merged = BatchNormalization()(merged)\n",
    "\n",
    "    merged = Dense(params['num_dense'], activation='relu')(merged)\n",
    "    merged = Dropout(params['dense_dropout_rate'])(merged)\n",
    "    merged = BatchNormalization()(merged)\n",
    "\n",
    "    output = Dense(1, activation='sigmoid')(merged)\n",
    "\n",
    "    model = Model(\n",
    "        inputs=[sequence_1_input, sequence_2_input],\n",
    "        outputs=output\n",
    "    )\n",
    "\n",
    "    model.compile(\n",
    "        loss=contrastive_loss,\n",
    "        optimizer='nadam',\n",
    "        metrics=['accuracy']\n",
    "    )\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def predict(model, X_q1, X_q2):\n",
    "    \"\"\"\n",
    "    Mirror the pairs, compute two separate predictions, and average them.\n",
    "    \"\"\"\n",
    "    \n",
    "    y1 = model.predict([X_q1, X_q2], batch_size=1024, verbose=1).reshape(-1)   \n",
    "    y2 = model.predict([X_q2, X_q1], batch_size=1024, verbose=1).reshape(-1)    \n",
    "    return (y1 + y2) / 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Partition the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "NUM_FOLDS = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "kfold = StratifiedKFold(\n",
    "    n_splits=NUM_FOLDS,\n",
    "    shuffle=True,\n",
    "    random_state=RANDOM_SEED\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create placeholders for out-of-fold predictions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_train_oofp = np.zeros_like(y_train, dtype='float64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_test_oofp = np.zeros((len(X_test_q1), NUM_FOLDS))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE = 2048"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "MAX_EPOCHS = 200"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Best values picked by Bayesian optimization."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_params = {\n",
    "    'dense_dropout_rate': 0.164,\n",
    "    'lstm_dropout_rate': 0.324,\n",
    "    'num_dense': 132,\n",
    "    'num_lstm': 254,\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The path where the best weights of the current model will be saved."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_checkpoint_path = project.temp_dir + 'fold-checkpoint-' + feature_list_id + '.h5'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fit the folds and compute out-of-fold predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Fitting fold 1 of 5\n",
      "\n",
      "Train on 646862 samples, validate on 161718 samples\n",
      "Epoch 1/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.1843 - acc: 0.7180Epoch 00000: val_loss improved from inf to 0.21473, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646862/646862 [==============================] - 79s - loss: 0.1842 - acc: 0.7181 - val_loss: 0.2147 - val_acc: 0.6840\n",
      "Epoch 2/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.1550 - acc: 0.7703Epoch 00001: val_loss improved from 0.21473 to 0.15562, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646862/646862 [==============================] - 77s - loss: 0.1550 - acc: 0.7703 - val_loss: 0.1556 - val_acc: 0.7790\n",
      "Epoch 3/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.1422 - acc: 0.7921Epoch 00002: val_loss improved from 0.15562 to 0.13599, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646862/646862 [==============================] - 78s - loss: 0.1422 - acc: 0.7920 - val_loss: 0.1360 - val_acc: 0.8013\n",
      "Epoch 4/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.1334 - acc: 0.8069Epoch 00003: val_loss improved from 0.13599 to 0.13269, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646862/646862 [==============================] - 78s - loss: 0.1335 - acc: 0.8069 - val_loss: 0.1327 - val_acc: 0.8066\n",
      "Epoch 5/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.1269 - acc: 0.8174Epoch 00004: val_loss improved from 0.13269 to 0.12810, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646862/646862 [==============================] - 77s - loss: 0.1268 - acc: 0.8174 - val_loss: 0.1281 - val_acc: 0.8144\n",
      "Epoch 6/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.1214 - acc: 0.8266Epoch 00005: val_loss improved from 0.12810 to 0.11976, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646862/646862 [==============================] - 78s - loss: 0.1214 - acc: 0.8265 - val_loss: 0.1198 - val_acc: 0.8286\n",
      "Epoch 7/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.1169 - acc: 0.8334Epoch 00006: val_loss improved from 0.11976 to 0.11909, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646862/646862 [==============================] - 78s - loss: 0.1169 - acc: 0.8334 - val_loss: 0.1191 - val_acc: 0.8290\n",
      "Epoch 8/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.1132 - acc: 0.8397Epoch 00007: val_loss did not improve\n",
      "646862/646862 [==============================] - 77s - loss: 0.1132 - acc: 0.8397 - val_loss: 0.1232 - val_acc: 0.8225\n",
      "Epoch 9/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.1097 - acc: 0.8450Epoch 00008: val_loss improved from 0.11909 to 0.11520, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646862/646862 [==============================] - 78s - loss: 0.1097 - acc: 0.8451 - val_loss: 0.1152 - val_acc: 0.8362\n",
      "Epoch 10/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.1069 - acc: 0.8500Epoch 00009: val_loss did not improve\n",
      "646862/646862 [==============================] - 78s - loss: 0.1069 - acc: 0.8500 - val_loss: 0.1170 - val_acc: 0.8345\n",
      "Epoch 11/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.1046 - acc: 0.8535Epoch 00010: val_loss did not improve\n",
      "646862/646862 [==============================] - 77s - loss: 0.1046 - acc: 0.8535 - val_loss: 0.1168 - val_acc: 0.8329\n",
      "Epoch 12/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.1021 - acc: 0.8574Epoch 00011: val_loss improved from 0.11520 to 0.11395, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646862/646862 [==============================] - 77s - loss: 0.1021 - acc: 0.8574 - val_loss: 0.1140 - val_acc: 0.8392\n",
      "Epoch 13/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.1002 - acc: 0.8601Epoch 00012: val_loss improved from 0.11395 to 0.11264, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646862/646862 [==============================] - 78s - loss: 0.1002 - acc: 0.8601 - val_loss: 0.1126 - val_acc: 0.8415\n",
      "Epoch 14/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.0985 - acc: 0.8630Epoch 00013: val_loss did not improve\n",
      "646862/646862 [==============================] - 78s - loss: 0.0985 - acc: 0.8631 - val_loss: 0.1137 - val_acc: 0.8396\n",
      "Epoch 15/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.0968 - acc: 0.8655Epoch 00014: val_loss improved from 0.11264 to 0.11197, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646862/646862 [==============================] - 79s - loss: 0.0968 - acc: 0.8655 - val_loss: 0.1120 - val_acc: 0.8426\n",
      "Epoch 16/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.0954 - acc: 0.8678Epoch 00015: val_loss did not improve\n",
      "646862/646862 [==============================] - 79s - loss: 0.0954 - acc: 0.8678 - val_loss: 0.1137 - val_acc: 0.8393\n",
      "Epoch 17/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.0941 - acc: 0.8696Epoch 00016: val_loss did not improve\n",
      "646862/646862 [==============================] - 79s - loss: 0.0941 - acc: 0.8696 - val_loss: 0.1126 - val_acc: 0.8423\n",
      "Epoch 00016: early stopping\n",
      "2344960/2345796 [============================>.] - ETA: 0s\n",
      "Fitting fold 2 of 5\n",
      "\n",
      "Train on 646862 samples, validate on 161718 samples\n",
      "Epoch 1/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.1846 - acc: 0.7179Epoch 00000: val_loss improved from inf to 0.20657, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646862/646862 [==============================] - 80s - loss: 0.1845 - acc: 0.7180 - val_loss: 0.2066 - val_acc: 0.6524\n",
      "Epoch 2/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.1547 - acc: 0.7701Epoch 00001: val_loss improved from 0.20657 to 0.14955, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646862/646862 [==============================] - 80s - loss: 0.1546 - acc: 0.7701 - val_loss: 0.1496 - val_acc: 0.7899\n",
      "Epoch 3/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.1417 - acc: 0.7918Epoch 00002: val_loss improved from 0.14955 to 0.13209, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646862/646862 [==============================] - 80s - loss: 0.1417 - acc: 0.7918 - val_loss: 0.1321 - val_acc: 0.8093\n",
      "Epoch 4/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.1331 - acc: 0.8071Epoch 00003: val_loss did not improve\n",
      "646862/646862 [==============================] - 74s - loss: 0.1331 - acc: 0.8071 - val_loss: 0.1323 - val_acc: 0.8092\n",
      "Epoch 5/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.1270 - acc: 0.8171Epoch 00004: val_loss improved from 0.13209 to 0.12600, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646862/646862 [==============================] - 71s - loss: 0.1270 - acc: 0.8171 - val_loss: 0.1260 - val_acc: 0.8192\n",
      "Epoch 6/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.1216 - acc: 0.8258Epoch 00005: val_loss improved from 0.12600 to 0.12143, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "646862/646862 [==============================] - 73s - loss: 0.1216 - acc: 0.8258 - val_loss: 0.1214 - val_acc: 0.8262\n",
      "Epoch 7/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.1173 - acc: 0.8333Epoch 00006: val_loss did not improve\n",
      "646862/646862 [==============================] - 71s - loss: 0.1173 - acc: 0.8333 - val_loss: 0.1222 - val_acc: 0.8253\n",
      "Epoch 8/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.1130 - acc: 0.8398- ETA: 1s - loss: 0.1130 - acc: Epoch 00007: val_loss improved from 0.12143 to 0.11808, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646862/646862 [==============================] - 71s - loss: 0.1130 - acc: 0.8399 - val_loss: 0.1181 - val_acc: 0.8325\n",
      "Epoch 9/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.1099 - acc: 0.8451Epoch 00008: val_loss improved from 0.11808 to 0.11705, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646862/646862 [==============================] - 71s - loss: 0.1099 - acc: 0.8451 - val_loss: 0.1170 - val_acc: 0.8340\n",
      "Epoch 10/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.1072 - acc: 0.8496Epoch 00009: val_loss did not improve\n",
      "646862/646862 [==============================] - 72s - loss: 0.1072 - acc: 0.8495 - val_loss: 0.1178 - val_acc: 0.8333\n",
      "Epoch 11/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.1044 - acc: 0.8538Epoch 00010: val_loss did not improve\n",
      "646862/646862 [==============================] - 77s - loss: 0.1044 - acc: 0.8538 - val_loss: 0.1172 - val_acc: 0.8350\n",
      "Epoch 12/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.1019 - acc: 0.8581Epoch 00011: val_loss improved from 0.11705 to 0.11340, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646862/646862 [==============================] - 71s - loss: 0.1019 - acc: 0.8581 - val_loss: 0.1134 - val_acc: 0.8402\n",
      "Epoch 13/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.1002 - acc: 0.8601- ETA: 1s - loss: 0.1002 - accEpoch 00012: val_loss improved from 0.11340 to 0.11280, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646862/646862 [==============================] - 74s - loss: 0.1002 - acc: 0.8602 - val_loss: 0.1128 - val_acc: 0.8414\n",
      "Epoch 14/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.0982 - acc: 0.8637- ETA: 5s -Epoch 00013: val_loss did not improve\n",
      "646862/646862 [==============================] - 72s - loss: 0.0982 - acc: 0.8636 - val_loss: 0.1153 - val_acc: 0.8390\n",
      "Epoch 15/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.0963 - acc: 0.8665Epoch 00014: val_loss did not improve\n",
      "646862/646862 [==============================] - 71s - loss: 0.0963 - acc: 0.8665 - val_loss: 0.1150 - val_acc: 0.8390\n",
      "Epoch 16/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.0951 - acc: 0.8687Epoch 00015: val_loss improved from 0.11280 to 0.11239, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646862/646862 [==============================] - 73s - loss: 0.0951 - acc: 0.8687 - val_loss: 0.1124 - val_acc: 0.8433\n",
      "Epoch 17/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.0942 - acc: 0.8698- ETAEpoch 00016: val_loss improved from 0.11239 to 0.11196, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646862/646862 [==============================] - 71s - loss: 0.0942 - acc: 0.8697 - val_loss: 0.1120 - val_acc: 0.8433\n",
      "Epoch 18/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.0925 - acc: 0.8724Epoch 00017: val_loss did not improve\n",
      "646862/646862 [==============================] - 72s - loss: 0.0925 - acc: 0.8725 - val_loss: 0.1122 - val_acc: 0.8439\n",
      "Epoch 19/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.0915 - acc: 0.8737Epoch 00018: val_loss improved from 0.11196 to 0.11112, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646862/646862 [==============================] - 73s - loss: 0.0915 - acc: 0.8736 - val_loss: 0.1111 - val_acc: 0.8454\n",
      "Epoch 20/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.0906 - acc: 0.875 - ETA: 0s - loss: 0.0906 - acc: 0.8754Epoch 00019: val_loss improved from 0.11112 to 0.10997, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646862/646862 [==============================] - 71s - loss: 0.0906 - acc: 0.8754 - val_loss: 0.1100 - val_acc: 0.8475\n",
      "Epoch 21/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.0892 - acc: 0.8773  E -Epoch 00020: val_loss improved from 0.10997 to 0.10888, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646862/646862 [==============================] - 72s - loss: 0.0892 - acc: 0.8773 - val_loss: 0.1089 - val_acc: 0.8493\n",
      "Epoch 22/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.0887 - acc: 0.8778Epoch 00021: val_loss did not improve\n",
      "646862/646862 [==============================] - 73s - loss: 0.0887 - acc: 0.8778 - val_loss: 0.1096 - val_acc: 0.8474\n",
      "Epoch 23/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.0878 - acc: 0.8793Epoch 00022: val_loss did not improve\n",
      "646862/646862 [==============================] - 72s - loss: 0.0878 - acc: 0.8793 - val_loss: 0.1107 - val_acc: 0.8466\n",
      "Epoch 24/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.0868 - acc: 0.8808Epoch 00023: val_loss did not improve\n",
      "646862/646862 [==============================] - 71s - loss: 0.0868 - acc: 0.8808 - val_loss: 0.1113 - val_acc: 0.8461\n",
      "Epoch 25/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.0863 - acc: 0.8818Epoch 00024: val_loss did not improve\n",
      "646862/646862 [==============================] - 71s - loss: 0.0863 - acc: 0.8818 - val_loss: 0.1101 - val_acc: 0.8475\n",
      "Epoch 00024: early stopping\n",
      "80859/80859 [==============================] - 3s     \n",
      "80859/80859 [==============================] - 3s     \n",
      "2345796/2345796 [==============================] - 106s   \n",
      "2345796/2345796 [==============================] - 112s   \n",
      "\n",
      "Fitting fold 3 of 5\n",
      "\n",
      "Train on 646864 samples, validate on 161716 samples\n",
      "Epoch 1/200\n",
      "645120/646864 [============================>.] - ETA: 0s - loss: 0.1856 - acc: 0.7159Epoch 00000: val_loss improved from inf to 0.20636, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646864/646864 [==============================] - 76s - loss: 0.1856 - acc: 0.7160 - val_loss: 0.2064 - val_acc: 0.6640\n",
      "Epoch 2/200\n",
      "645120/646864 [============================>.] - ETA: 0s - loss: 0.1556 - acc: 0.7685Epoch 00001: val_loss improved from 0.20636 to 0.16753, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646864/646864 [==============================] - 80s - loss: 0.1555 - acc: 0.7686 - val_loss: 0.1675 - val_acc: 0.7489\n",
      "Epoch 3/200\n",
      "645120/646864 [============================>.] - ETA: 0s - loss: 0.1423 - acc: 0.7911Epoch 00002: val_loss improved from 0.16753 to 0.13742, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646864/646864 [==============================] - 76s - loss: 0.1423 - acc: 0.7911 - val_loss: 0.1374 - val_acc: 0.8001\n",
      "Epoch 4/200\n",
      "645120/646864 [============================>.] - ETA: 0s - loss: 0.1332 - acc: 0.8070Epoch 00003: val_loss improved from 0.13742 to 0.13103, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "646864/646864 [==============================] - 72s - loss: 0.1331 - acc: 0.8070 - val_loss: 0.1310 - val_acc: 0.8093\n",
      "Epoch 5/200\n",
      "645120/646864 [============================>.] - ETA: 0s - loss: 0.1265 - acc: 0.8181Epoch 00004: val_loss improved from 0.13103 to 0.12388, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646864/646864 [==============================] - 72s - loss: 0.1265 - acc: 0.8181 - val_loss: 0.1239 - val_acc: 0.8209\n",
      "Epoch 6/200\n",
      "645120/646864 [============================>.] - ETA: 0s - loss: 0.1211 - acc: 0.8266Epoch 00005: val_loss improved from 0.12388 to 0.12240, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646864/646864 [==============================] - 71s - loss: 0.1211 - acc: 0.8267 - val_loss: 0.1224 - val_acc: 0.8248\n",
      "Epoch 7/200\n",
      "645120/646864 [============================>.] - ETA: 0s - loss: 0.1167 - acc: 0.8340Epoch 00006: val_loss did not improve\n",
      "646864/646864 [==============================] - 72s - loss: 0.1167 - acc: 0.8340 - val_loss: 0.1245 - val_acc: 0.8199\n",
      "Epoch 8/200\n",
      "645120/646864 [============================>.] - ETA: 0s - loss: 0.1128 - acc: 0.8403Epoch 00007: val_loss improved from 0.12240 to 0.11539, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646864/646864 [==============================] - 79s - loss: 0.1128 - acc: 0.8402 - val_loss: 0.1154 - val_acc: 0.8355\n",
      "Epoch 9/200\n",
      "645120/646864 [============================>.] - ETA: 0s - loss: 0.1092 - acc: 0.8460Epoch 00008: val_loss did not improve\n",
      "646864/646864 [==============================] - 72s - loss: 0.1092 - acc: 0.8460 - val_loss: 0.1162 - val_acc: 0.8362\n",
      "Epoch 10/200\n",
      "645120/646864 [============================>.] - ETA: 0s - loss: 0.1068 - acc: 0.8500Epoch 00009: val_loss improved from 0.11539 to 0.11494, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646864/646864 [==============================] - 72s - loss: 0.1067 - acc: 0.8500 - val_loss: 0.1149 - val_acc: 0.8379\n",
      "Epoch 11/200\n",
      "645120/646864 [============================>.] - ETA: 0s - loss: 0.1038 - acc: 0.8549- ETA: 2s - loss: 0.1038 - aEpoch 00010: val_loss improved from 0.11494 to 0.11353, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646864/646864 [==============================] - 72s - loss: 0.1038 - acc: 0.8549 - val_loss: 0.1135 - val_acc: 0.8395\n",
      "Epoch 12/200\n",
      "645120/646864 [============================>.] - ETA: 0s - loss: 0.1018 - acc: 0.8578Epoch 00011: val_loss improved from 0.11353 to 0.11215, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646864/646864 [==============================] - 71s - loss: 0.1018 - acc: 0.8578 - val_loss: 0.1121 - val_acc: 0.8427\n",
      "Epoch 13/200\n",
      "645120/646864 [============================>.] - ETA: 0s - loss: 0.0997 - acc: 0.8609Epoch 00012: val_loss improved from 0.11215 to 0.11115, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646864/646864 [==============================] - 71s - loss: 0.0997 - acc: 0.8609 - val_loss: 0.1112 - val_acc: 0.8446\n",
      "Epoch 14/200\n",
      "645120/646864 [============================>.] - ETA: 0s - loss: 0.0979 - acc: 0.8641Epoch 00013: val_loss did not improve\n",
      "646864/646864 [==============================] - 73s - loss: 0.0979 - acc: 0.8641 - val_loss: 0.1153 - val_acc: 0.8383\n",
      "Epoch 15/200\n",
      "645120/646864 [============================>.] - ETA: 0s - loss: 0.0965 - acc: 0.8662Epoch 00014: val_loss did not improve\n",
      "646864/646864 [==============================] - 80s - loss: 0.0965 - acc: 0.8662 - val_loss: 0.1172 - val_acc: 0.8369\n",
      "Epoch 16/200\n",
      "645120/646864 [============================>.] - ETA: 0s - loss: 0.0951 - acc: 0.8685Epoch 00015: val_loss did not improve\n",
      "646864/646864 [==============================] - 81s - loss: 0.0951 - acc: 0.8685 - val_loss: 0.1131 - val_acc: 0.8412\n",
      "Epoch 00015: early stopping\n",
      "80858/80858 [==============================] - 4s     \n",
      "2344960/2345796 [============================>.] - ETA: 0s\n",
      "Fitting fold 4 of 5\n",
      "\n",
      "Train on 646866 samples, validate on 161714 samples\n",
      "Epoch 1/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.1865 - acc: 0.7153Epoch 00000: val_loss improved from inf to 0.20801, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646866/646866 [==============================] - 88s - loss: 0.1864 - acc: 0.7154 - val_loss: 0.2080 - val_acc: 0.6537\n",
      "Epoch 2/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.1565 - acc: 0.7676Epoch 00001: val_loss improved from 0.20801 to 0.15519, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646866/646866 [==============================] - 86s - loss: 0.1565 - acc: 0.7676 - val_loss: 0.1552 - val_acc: 0.7859\n",
      "Epoch 3/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.1428 - acc: 0.7911Epoch 00002: val_loss improved from 0.15519 to 0.13651, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646866/646866 [==============================] - 85s - loss: 0.1428 - acc: 0.7912 - val_loss: 0.1365 - val_acc: 0.8009\n",
      "Epoch 4/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.1343 - acc: 0.8054Epoch 00003: val_loss improved from 0.13651 to 0.12605, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646866/646866 [==============================] - 73s - loss: 0.1343 - acc: 0.8053 - val_loss: 0.1261 - val_acc: 0.8188\n",
      "Epoch 5/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.1277 - acc: 0.8160Epoch 00004: val_loss improved from 0.12605 to 0.12560, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646866/646866 [==============================] - 77s - loss: 0.1277 - acc: 0.8160 - val_loss: 0.1256 - val_acc: 0.8183\n",
      "Epoch 6/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.1220 - acc: 0.8260Epoch 00005: val_loss improved from 0.12560 to 0.12178, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646866/646866 [==============================] - 77s - loss: 0.1221 - acc: 0.8260 - val_loss: 0.1218 - val_acc: 0.8252\n",
      "Epoch 7/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.1175 - acc: 0.8333Epoch 00006: val_loss improved from 0.12178 to 0.12102, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646866/646866 [==============================] - 77s - loss: 0.1176 - acc: 0.8332 - val_loss: 0.1210 - val_acc: 0.8272\n",
      "Epoch 8/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.1137 - acc: 0.8394Epoch 00007: val_loss improved from 0.12102 to 0.12044, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646866/646866 [==============================] - 86s - loss: 0.1137 - acc: 0.8394 - val_loss: 0.1204 - val_acc: 0.8281\n",
      "Epoch 9/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.1098 - acc: 0.8455Epoch 00008: val_loss improved from 0.12044 to 0.11472, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646866/646866 [==============================] - 76s - loss: 0.1099 - acc: 0.8454 - val_loss: 0.1147 - val_acc: 0.8376\n",
      "Epoch 10/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.1073 - acc: 0.8490Epoch 00009: val_loss improved from 0.11472 to 0.11432, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "646866/646866 [==============================] - 85s - loss: 0.1073 - acc: 0.8490 - val_loss: 0.1143 - val_acc: 0.8370\n",
      "Epoch 11/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.1046 - acc: 0.8539Epoch 00010: val_loss did not improve\n",
      "646866/646866 [==============================] - 80s - loss: 0.1045 - acc: 0.8539 - val_loss: 0.1170 - val_acc: 0.8352\n",
      "Epoch 12/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.1026 - acc: 0.8568Epoch 00011: val_loss improved from 0.11432 to 0.11395, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646866/646866 [==============================] - 86s - loss: 0.1026 - acc: 0.8568 - val_loss: 0.1140 - val_acc: 0.8396\n",
      "Epoch 13/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.1003 - acc: 0.8604Epoch 00012: val_loss did not improve\n",
      "646866/646866 [==============================] - 86s - loss: 0.1003 - acc: 0.8604 - val_loss: 0.1147 - val_acc: 0.8386\n",
      "Epoch 00012: early stopping\n",
      "80857/80857 [==============================] - 4s     \n",
      "80857/80857 [==============================] - 4s     \n",
      "2345796/2345796 [==============================] - 124s   \n",
      "\n",
      "Fitting fold 5 of 5\n",
      "\n",
      "Train on 646866 samples, validate on 161714 samples\n",
      "Epoch 1/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.1840 - acc: 0.7181Epoch 00000: val_loss improved from inf to 0.21199, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646866/646866 [==============================] - 86s - loss: 0.1839 - acc: 0.7182 - val_loss: 0.2120 - val_acc: 0.6538\n",
      "Epoch 2/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.1550 - acc: 0.7697Epoch 00001: val_loss improved from 0.21199 to 0.15076, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646866/646866 [==============================] - 86s - loss: 0.1550 - acc: 0.7698 - val_loss: 0.1508 - val_acc: 0.7914\n",
      "Epoch 3/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.1420 - acc: 0.7920Epoch 00002: val_loss improved from 0.15076 to 0.13195, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646866/646866 [==============================] - 86s - loss: 0.1420 - acc: 0.7920 - val_loss: 0.1319 - val_acc: 0.8077\n",
      "Epoch 4/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.1335 - acc: 0.8062Epoch 00003: val_loss improved from 0.13195 to 0.12728, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646866/646866 [==============================] - 85s - loss: 0.1335 - acc: 0.8061 - val_loss: 0.1273 - val_acc: 0.8165\n",
      "Epoch 5/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.1270 - acc: 0.8172Epoch 00004: val_loss improved from 0.12728 to 0.12308, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646866/646866 [==============================] - 85s - loss: 0.1270 - acc: 0.8172 - val_loss: 0.1231 - val_acc: 0.8233\n",
      "Epoch 6/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.1213 - acc: 0.8263Epoch 00005: val_loss improved from 0.12308 to 0.12024, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646866/646866 [==============================] - 86s - loss: 0.1213 - acc: 0.8263 - val_loss: 0.1202 - val_acc: 0.8273\n",
      "Epoch 7/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.1170 - acc: 0.8334Epoch 00006: val_loss did not improve\n",
      "646866/646866 [==============================] - 85s - loss: 0.1170 - acc: 0.8335 - val_loss: 0.1211 - val_acc: 0.8270\n",
      "Epoch 8/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.1133 - acc: 0.8397Epoch 00007: val_loss improved from 0.12024 to 0.11713, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646866/646866 [==============================] - 85s - loss: 0.1133 - acc: 0.8397 - val_loss: 0.1171 - val_acc: 0.8322\n",
      "Epoch 9/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.1100 - acc: 0.8452Epoch 00008: val_loss did not improve\n",
      "646866/646866 [==============================] - 85s - loss: 0.1101 - acc: 0.8452 - val_loss: 0.1181 - val_acc: 0.8322\n",
      "Epoch 10/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.1073 - acc: 0.8490Epoch 00009: val_loss improved from 0.11713 to 0.11526, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646866/646866 [==============================] - 85s - loss: 0.1073 - acc: 0.8490 - val_loss: 0.1153 - val_acc: 0.8365\n",
      "Epoch 11/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.1048 - acc: 0.8528Epoch 00010: val_loss improved from 0.11526 to 0.11182, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646866/646866 [==============================] - 86s - loss: 0.1048 - acc: 0.8527 - val_loss: 0.1118 - val_acc: 0.8421\n",
      "Epoch 12/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.1026 - acc: 0.8564Epoch 00011: val_loss did not improve\n",
      "646866/646866 [==============================] - 85s - loss: 0.1026 - acc: 0.8565 - val_loss: 0.1146 - val_acc: 0.8380\n",
      "Epoch 13/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.1004 - acc: 0.8597Epoch 00012: val_loss did not improve\n",
      "646866/646866 [==============================] - 85s - loss: 0.1004 - acc: 0.8597 - val_loss: 0.1136 - val_acc: 0.8399\n",
      "Epoch 14/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.0987 - acc: 0.8624Epoch 00013: val_loss improved from 0.11182 to 0.11155, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646866/646866 [==============================] - 85s - loss: 0.0987 - acc: 0.8624 - val_loss: 0.1116 - val_acc: 0.8433\n",
      "Epoch 15/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.0969 - acc: 0.8651Epoch 00014: val_loss improved from 0.11155 to 0.11119, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_siamese_lstm_attention.h5\n",
      "646866/646866 [==============================] - 86s - loss: 0.0969 - acc: 0.8650 - val_loss: 0.1112 - val_acc: 0.8441\n",
      "Epoch 00014: early stopping\n",
      "80857/80857 [==============================] - 4s     \n",
      "80857/80857 [==============================] - 4s     \n",
      "2344960/2345796 [============================>.] - ETA: 0sCPU times: user 1h 40min 1s, sys: 19min 13s, total: 1h 59min 15s\n",
      "Wall time: 2h 13min 13s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# Iterate through folds.\n",
    "for fold_num, (ix_train, ix_val) in enumerate(kfold.split(X_train_q1, y_train)):\n",
    "    \n",
    "    # Augment the training set by mirroring the pairs.\n",
    "    X_fold_train_q1 = np.vstack([X_train_q1[ix_train], X_train_q2[ix_train]])\n",
    "    X_fold_train_q2 = np.vstack([X_train_q2[ix_train], X_train_q1[ix_train]])\n",
    "\n",
    "    X_fold_val_q1 = np.vstack([X_train_q1[ix_val], X_train_q2[ix_val]])\n",
    "    X_fold_val_q2 = np.vstack([X_train_q2[ix_val], X_train_q1[ix_val]])\n",
    "\n",
    "    # Ground truth should also be \"mirrored\".\n",
    "    y_fold_train = np.concatenate([y_train[ix_train], y_train[ix_train]])\n",
    "    y_fold_val = np.concatenate([y_train[ix_val], y_train[ix_val]])\n",
    "    \n",
    "    print()\n",
    "    print(f'Fitting fold {fold_num + 1} of {kfold.n_splits}')\n",
    "    print()\n",
    "    \n",
    "    # Compile a new model.\n",
    "    model = create_model(model_params)\n",
    "\n",
    "    # Train.\n",
    "    model.fit(\n",
    "        [X_fold_train_q1, X_fold_train_q2], y_fold_train,\n",
    "        validation_data=([X_fold_val_q1, X_fold_val_q2], y_fold_val),\n",
    "\n",
    "        batch_size=BATCH_SIZE,\n",
    "        epochs=MAX_EPOCHS,\n",
    "        verbose=1,\n",
    "        \n",
    "        callbacks=[\n",
    "            # Stop training when the validation loss stops improving.\n",
    "            EarlyStopping(\n",
    "                monitor='val_loss',\n",
    "                min_delta=0.001,\n",
    "                patience=3,\n",
    "                verbose=1,\n",
    "                mode='auto',\n",
    "            ),\n",
    "            # Save the weights of the best epoch.\n",
    "            ModelCheckpoint(\n",
    "                model_checkpoint_path,\n",
    "                monitor='val_loss',\n",
    "                save_best_only=True,\n",
    "                verbose=2,\n",
    "            ),\n",
    "        ],\n",
    "    )\n",
    "        \n",
    "    # Restore the best epoch.\n",
    "    model.load_weights(model_checkpoint_path)\n",
    "    \n",
    "    # Compute out-of-fold predictions.\n",
    "    y_train_oofp[ix_val] = predict(model, X_train_q1[ix_val], X_train_q2[ix_val])\n",
    "    y_test_oofp[:, fold_num] = predict(model, X_test_q1, X_test_q2)\n",
    "    \n",
    "    # Clear GPU memory.\n",
    "    K.clear_session()\n",
    "    del X_fold_train_q1\n",
    "    del X_fold_train_q2\n",
    "    del X_fold_val_q1\n",
    "    del X_fold_val_q2\n",
    "    del model\n",
    "    gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CV score: 0.360265255459\n"
     ]
    }
   ],
   "source": [
    "cv_score = log_loss(y_train, y_train_oofp)\n",
    "print('CV score:', cv_score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features_train = y_train_oofp.reshape((-1, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features_test = np.mean(y_test_oofp, axis=1).reshape((-1, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X train: (404290, 1)\n",
      "X test:  (2345796, 1)\n"
     ]
    }
   ],
   "source": [
    "print('X train:', features_train.shape)\n",
    "print('X test: ', features_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "feature_names = [feature_list_id]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "project.save_features(features_train, features_test, feature_names, feature_list_id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Explore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f6fcfa06940>"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqAAAAGoCAYAAACZh1c1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X90lOWZ//HPTBIyM8RCQkBLS2MI7YItJBBAoikgWeu2\nJkQhPWoUKS6aIMraClVKMdhAaVeSXaldBA54SqBlCy4EIroUFYSFVBApFcSqmapbFs0YiTKZ/DDz\nfP/wMF+nQBhJcs+PvF/neHry3M/MdU0vJ3y8H+YZm2VZlgAAAABD7OFuAAAAAD0LARQAAABGEUAB\nAABgFAEUAAAARhFAAQAAYBQBFAAAAEbFh7uBWFRf/0m317DZbOrXr7c+/NAr7qQVuZhT5GNG0YE5\nRT5mFB1Mzql//8suuMYOaJSy2z/7l8jOBCMac4p8zCg6MKfIx4yiQ6TMiX9NAAAAYBQBFAAAAEYR\nQAEAAGAUARQAAABGEUABAABgFAEUAAAARhFAAQAAYBQBFAAAAEYRQAEAAGAUARQAAABG8V3wAAAA\nYXLXL14wUmftw5O+0Pl/+csJPfbYz+V21+mrX/2a5s6dr299a3iX9cMOKAAAAAJaWlr00EM/0ve+\nN1nPPbdbRUW36OGHf6SmpqYuq0EABQAAQMDhw4dks9l0881Fio+PV35+oVJSUnTgwP90WQ0CKAAA\nAALeffevuvLKwUHHvva1NL377l+7rAYBFAAAAAE+n08OhyPoWGKiQ83NzV1WgwAKAACAAIfDoZaW\nlqBjLS3NcjqdXVaDAAoAAICAtLR0vfvuO0HH3n33HaWnD77AI744bsMUxQoerA53C13ui94mAgAA\ndK3s7DFqa2vV5s0bddNNRXruuWfU0NCgsWNzuqwGO6AAAAAI6NWrl5YtW65du3bqu9+dpKef/k/9\n4heVXXoJnh1QAACAMInUK39DhnxdTz65ttuenx1QAAAAGEUABQAAgFEEUAAAABhFAAUAAIBRBFAA\nAAAYRQAFAACAUQRQAAAAGEUABQAAgFEEUAAAABhFAAUAAIBRBFAAAAAYRQAFAACAUQRQAAAAGEUA\nBQAAgFEEUAAAABhFAAUAAIBRBFAAAAAYRQAFAACAUQRQAAAAGEUABQAAgFEEUAAAABgVlgB69OhR\n5ebmBn4+deqU7r33Xl199dW69tprVV5ertbWVkmSZVmqqKjQuHHjNGbMGC1evFjt7e2Bx9bU1Cgv\nL09ZWVkqKSmRx+MJrB0/flxFRUXKyspSYWGhjhw5ElhrbGzU7NmzlZ2drYkTJ2rTpk2BtYvVBAAA\nwKUzGkAty9LmzZt11113qa2tLXB83rx5uuKKK/TSSy9p69at+vOf/6xf//rXkqQNGzZo9+7d2rZt\nm3bs2KHDhw9r7dq1kqQTJ06orKxMlZWVqq2tVWpqqubPny9JamlpUWlpqaZMmaKDBw9q2rRpmjVr\nlrxeryRp4cKFcrlc2r9/v5YvX65ly5YFAmpHNQEAANA5RgPok08+qXXr1qm0tDRwrLW1VU6nU7Nm\nzVJiYqL69++vgoICvfrqq5Kk6upqTZ8+XQMGDFD//v1VUlKiLVu2SJK2b9+uvLw8ZWZmyuFwaO7c\nudq7d688Ho9qa2tlt9tVXFyshIQEFRUVKTU1VXv27JHX69WuXbs0Z84cJSYmasSIEcrPz9fWrVsv\nWhMAAACdE2+y2NSpU1VaWqqXX345cKxXr15atWpV0Hkvvviihg4dKkmqq6vTkCFDAmvp6elyu92y\nLEt1dXUaOXJkYC05OVl9+vSR2+2W2+1WRkZG0POmp6errq5OV155peLj4zVo0KCgtZ07d160ps1m\nu+jrtNlssndztLfbL95HNIqLi63XdXZOsTqvWMCMogNzinzMKDpEypyMBtABAwZ0uG5ZlpYsWaK6\nujo99thjkiSfzyeHwxE4x+l0yu/3q7W19Zy1s+s+n09NTU1yOp1Baw6HQ83NzWpqajrncWfXLlYz\nMTHxoq+zX7/eIQVVnCslJSncLXSLvn17h7sFXAQzig7MKfIxo+gQ7jkZDaAdaW5u1o9//GO98cYb\nqqqqUr9+/SR9FgxbWloC5/l8PsXHxysxMTEoNH5+3eVyyel0nrPW3NwcWPv8c35+7WI1Q/Hhh152\nQC9RQ8OZcLfQpex2m/r27a3Tp73y+61wt4PzYEbRgTlFPmYUHUzOqaNNpYgIoKdPn9bMmTPlcrn0\nn//5n+rbt29gLSMjQ263W5mZmZIkt9utwYMHB62d1dDQoMbGRmVkZMjr9Wr9+vVBddxut/Lz85WW\nlqa2tjadPHlSAwcODKydvezeUc1QWJYlPjR/adrbY/OXlt9vxexrixXMKDowp8jHjKJDuOcU9vuA\nWpal+++/X6mpqVqzZk1Q+JSkyZMna82aNTp16pQ8Ho9WrlypwsJCSVJ+fr527typQ4cOqaWlRZWV\nlRo/frySk5OVk5Oj1tZWVVVVqa2tTZs3b5bH41Fubq6SkpKUl5eniooK+Xw+HT16VDU1NSooKLho\nTQAAAHRO2HdAX331Vb388stKTEzU2LFjA8evuuoqbdiwQcXFxfJ4PCoqKlJbW5sKCgo0Y8YMSdKw\nYcNUXl6uBQsWqL6+XqNHj9bSpUslffbhptWrV2vRokWqrKxUWlqaVqxYEbjMXl5errKyMk2YMEEu\nl0vz5s0L7Hh2VBMAAACdY7Msi33yLlZf/0m314iLs2n6kue7vY5pax+eFO4WulRcnE0pKUlqaDjD\nJakIxYyiA3OKfMwoOpicU//+l11wLeyX4AEAANCzEEABAABgFAEUAAAARhFAAQAAYBQBFAAAAEYR\nQAEAAGAUARQAAABGEUABAABgFAEUAAAARhFAAQAAYBQBFAAAAEYRQAEAAGAUARQAAABGEUABAABg\nFAEUAAAARhFAAQAAYBQBFAAAAEYRQAEAAGAUARQAAABGEUABAABgFAEUAAAARhFAAQAAYBQBFAAA\nAEYRQAEAAGAUARQAAABGEUABAABgFAEUAAAARhFAAQAAYBQBFAAAAEYRQAEAAGAUARQAAABGEUAB\nAABgFAEUAAAARhFAAQAAYBQBFAAAAEYRQAEAAGAUARQAAABGEUABAABgFAEUAAAARhFAAQAAYBQB\nFAAAAEYRQAEAAGAUARQAAABGEUABAABgFAEUAAAARhFAAQAAYBQBFAAAAEaFJYAePXpUubm5gZ8b\nGxs1e/ZsZWdna+LEidq0aVNgzbIsVVRUaNy4cRozZowWL16s9vb2wHpNTY3y8vKUlZWlkpISeTye\nwNrx48dVVFSkrKwsFRYW6siRI11SEwAAAJfOaAC1LEubN2/WXXfdpba2tsDxhQsXyuVyaf/+/Vq+\nfLmWLVsWCIsbNmzQ7t27tW3bNu3YsUOHDx/W2rVrJUknTpxQWVmZKisrVVtbq9TUVM2fP1+S1NLS\notLSUk2ZMkUHDx7UtGnTNGvWLHm93k7VBAAAQOcYDaBPPvmk1q1bp9LS0sAxr9erXbt2ac6cOUpM\nTNSIESOUn5+vrVu3SpKqq6s1ffp0DRgwQP3791dJSYm2bNkiSdq+fbvy8vKUmZkph8OhuXPnau/e\nvfJ4PKqtrZXdbldxcbESEhJUVFSk1NRU7dmzp1M1AQAA0DnxJotNnTpVpaWlevnllwPH3nnnHcXH\nx2vQoEGBY+np6dq5c6ckqa6uTkOGDAlac7vdsixLdXV1GjlyZGAtOTlZffr0kdvtltvtVkZGRlD9\n9PR01dXV6corr7zkmjab7aKv02azyd7N0d5uv3gf0SguLrZe19k5xeq8YgEzig7MKfIxo+gQKXMy\nGkAHDBhwzrGmpiY5HI6gYw6HQ83NzZIkn88XtO50OuX3+9Xa2nrO2tl1n8+npqYmOZ3O8z5vZ2om\nJiZe9HX269c7pKCKc6WkJIW7hW7Rt2/vcLeAi2BG0YE5RT5mFB3CPSejAfR8nE6nWlpago41NzfL\n5XJJ+iwYfn7d5/MpPj5eiYmJQaHx8+sul0tOp/OctbPP25maofjwQy87oJeooeFMuFvoUna7TX37\n9tbp0175/Va428F5MKPowJwiHzOKDibn1NGmUtgDaFpamtra2nTy5EkNHDhQkuR2uwOXwDMyMuR2\nu5WZmRlYGzx4cNDaWQ0NDWpsbFRGRoa8Xq/Wr18fVMvtdis/P79TNUNhWZb40PylaW+PzV9afr8V\ns68tVjCj6MCcIh8zig7hnlPY7wOalJSkvLw8VVRUyOfz6ejRo6qpqVFBQYEkafLkyVqzZo1OnTol\nj8ejlStXqrCwUJKUn5+vnTt36tChQ2ppaVFlZaXGjx+v5ORk5eTkqLW1VVVVVWpra9PmzZvl8XiU\nm5vbqZoAAADonLDvgEpSeXm5ysrKNGHCBLlcLs2bNy+w+1hcXCyPx6OioiK1tbWpoKBAM2bMkCQN\nGzZM5eXlWrBggerr6zV69GgtXbpUktSrVy+tXr1aixYtUmVlpdLS0rRixYrAZfZLrQkAAIDOsVmW\nxT55F6uv/6Tba8TF2TR9yfPdXse0tQ9PCncLXSouzqaUlCQ1NJzhklSEYkbRgTlFPmYUHUzOqX//\nyy64FvZL8AAAAOhZCKAAAAAwigAKAAAAowigAAAAMIoACgAAAKMIoAAAADCKAAoAAACjCKAAAAAw\nigAKAAAAowigAAAAMIoACgAAAKMIoAAAADCKAAoAAACjCKAAAAAwigAKAAAAowigAAAAMIoACgAA\nAKMIoAAAADCKAAoAAACjCKAAAAAwigAKAAAAowigAAAAMIoACgAAAKMIoAAAADCKAAoAAACjCKAA\nAAAwigAKAAAAowigAAAAMIoACgAAAKMIoAAAADCKAAoAAACjCKAAAAAwigAKAAAAowigAAAAMIoA\nCgAAAKMIoAAAADCKAAoAAACjCKAAAAAwigAKAAAAowigAAAAMIoACgAAAKMIoAAAADCKAAoAAACj\nCKAAAAAwigAKAAAAowigAAAAMIoACgAAAKMiJoAePnxYU6ZM0ahRo3TDDTdo+/btkqTGxkbNnj1b\n2dnZmjhxojZt2hR4jGVZqqio0Lhx4zRmzBgtXrxY7e3tgfWamhrl5eUpKytLJSUl8ng8gbXjx4+r\nqKhIWVlZKiws1JEjRwJrHdUEAABA50REAG1vb9fs2bN1zz336PDhw1qyZIkefvhh/e///q8WLlwo\nl8ul/fv3a/ny5Vq2bFkgLG7YsEG7d+/Wtm3btGPHDh0+fFhr166VJJ04cUJlZWWqrKxUbW2tUlNT\nNX/+fElSS0uLSktLNWXKFB08eFDTpk3TrFmz5PV6JanDmgAAAOiciAigH3/8sRoaGtTe3i7LsmSz\n2ZSQkKC4uDjt2rVLc+bMUWJiokaMGKH8/Hxt3bpVklRdXa3p06drwIAB6t+/v0pKSrRlyxZJ0vbt\n25WXl6fMzEw5HA7NnTtXe/fulcfjUW1trex2u4qLi5WQkKCioiKlpqZqz5498nq9HdYEAABA58SH\nuwFJSk5OVnFxsX70ox9p3rx58vv9WrJkiT766CPFx8dr0KBBgXPT09O1c+dOSVJdXZ2GDBkStOZ2\nu2VZlurq6jRy5MigGn369JHb7Zbb7VZGRkZQD+np6aqrq9OVV17ZYc1Q2Gw22bs52tvttu4tECZx\ncbH1us7OKVbnFQuYUXRgTpGPGUWHSJlTRARQv98vh8Ohxx9/XJMmTdL+/fv14IMPasWKFXI4HEHn\nOhwONTc3S5J8Pl/QutPplN/vV2tr6zlrZ9d9Pp+amprkdDrP+7xNTU0d1gxFv369ZbPxBrwUKSlJ\n4W6hW/Tt2zvcLeAimFF0YE6RjxlFh3DPKSIC6M6dO3X06FE99NBDkqSJEydq4sSJ+tWvfqWWlpag\nc5ubm+VyuSR9Fgw/v+7z+RQfH6/ExMTzhkafzyeXyyWn03nO2tnndTqdHdYMxYcfetkBvUQNDWfC\n3UKXsttt6tu3t06f9srvt8LdDs6DGUUH5hT5mFF0MDmnjjaVIiKA/t///Z9aW1uDjsXHx+ub3/ym\nXnnlFZ08eVIDBw6UJLnd7sBl94yMDLndbmVmZgbWBg8eHLR2VkNDgxobG5WRkSGv16v169cH1XO7\n3crPz1daWpra2touWDMUlmXpcx/GxxfQ3h6bv7T8fitmX1usYEbRgTlFPmYUHcI9p5D36QoLC7Vm\nzRqdOnWqy5u45ppr9Prrr+vpp5+WZVl6+eWX9Yc//EE33nij8vLyVFFRIZ/Pp6NHj6qmpkYFBQWS\npMmTJwd68ng8WrlypQoLCyVJ+fn52rlzpw4dOqSWlhZVVlZq/PjxSk5OVk5OjlpbW1VVVaW2tjZt\n3rxZHo9Hubm5SkpK6rAmAAAAOsdmWVZI8Xf9+vV65plndPToUY0aNUr5+fn6p3/6J/Xp06dLGnnh\nhRf0+OOP67333tPAgQP1L//yL7r++ut1+vRplZWV6cCBA3K5XLrvvvtUVFQk6bPbNy1fvlxPP/20\n2traVFBQoPnz5ysuLk6StGPHDj3++OOqr6/X6NGjtXTpUvXr10/SZ7dpWrRokd544w2lpaVp0aJF\nysrKkqQOa4aivv6TLvn/pCNxcTZNX/J8t9cxbe3Dk8LdQpeKi7MpJSVJDQ1n2BGIUMwoOjCnyMeM\nooPJOfXvf9kF10IOoGf97W9/044dO/Tss8/qrbfeUm5urgoKCpSXl6devXp1utlYQAC9dARQmMaM\nogNzinzMKDpESgD9wn8H9Ctf+YruvvtuTZ48WRs3btTatWv1wgsvKCkpSTfddJPuv//+LtsVBQAA\nQOz5Qp/V9ng8Wr9+vYqLi3Xddddpz549euCBB/TSSy/pN7/5jY4fP67S0tLu6hUAAAAxIOQd0Dvv\nvFOvvPKKrrjiCuXn56u8vDzoZu4DBgzQnXfeqQULFnRLowAAAIgNIQfQjIwMPfDAAxo1atQFzxk7\ndixfWQkAAIAOhXwJvqysTPX19XrxxRcDxxYuXKhdu3YFfk5JSQn6CksAAADg74UcQJ966inNnz9f\np0+fDhz70pe+pIceekgbN27sluYAAAAQe0IOoFVVVaqoqNDNN98cODZv3jz98pe/1Jo1a7qlOQAA\nAMSekAPoRx99pLS0tHOODxkyRB988EGXNgUAAIDYFXIAzczM1Jo1a9T+uS85tyxL69at01VXXdUt\nzQEAACD2hPwp+Icfflg/+MEPtG/fPg0bNkyS9MYbb6i1tVWrVq3qtgYBAAAQW0IOoEOHDtWzzz6r\nHTt26O2331ZCQoImTJiggoICJSUldWePAAAAiCFf6Ks4k5OTdfvtt3dXLwAAAOgBQg6g7777rpYt\nW6bXXntNbW1tsqzgL7Dft29flzcHAACA2BNyAJ0/f74aGho0Y8YMLrkDAADgkoUcQP/85z9r8+bN\n+sY3vtGd/QAAACDGhXwbpoEDB+rMmTPd2QsAAAB6gJB3QB988EE9+uijuu+++5SWlqaEhISg9fT0\n9C5vDgAAALEn5AB6//33B/2vJNlsNlmWJZvNptdff73ruwMAAEDMCTmAPv/8893ZBwAAAHqIkAPo\nV77yFUnS+++/L7fbraysLJ05c0apqand1hwAAABiT8gfQmpqatIDDzygCRMm6K677lJ9fb0eeeQR\nFRcXq6GhoTt7BAAAQAwJOYA+9thjev/99/Xss88qMTFR0mcfTGppadHPf/7zbmsQAAAAsSXkAPr8\n889r/vz5QZ92z8jI0KOPPqq9e/d2S3MAAACIPSEH0DNnzpz3G5Dsdrs+/fTTLm0KAAAAsSvkAJqb\nm6snn3xS7e3tgWMfffSRHnvsMV177bXd0hwAAABiT8gB9Kc//an++te/KicnR83NzZo5c6auu+46\nNTY2asGCBd3ZIwAAAGJIyLdhGjBggH7/+9/rwIEDqqur06effqqMjAxde+21stls3dkjAAAAYkjI\nAfSsnJwc5eTkdEcvAAAA6AFCDqBDhw7tcKeTr+IEAABAKEIOoKtXrw76ub29Xe+++66qqqr0wx/+\nsMsbAwAAQGwKOYB++9vfPu/xIUOGqKKiQt/73ve6rCkAAADErpA/BX8hX/7yl/Xmm292RS8AAADo\nAULeAd23b985x86cOaMNGzZo6NChXdoUAAAAYlfIAXTmzJnnHEtISNDw4cP1s5/9rEubAgAAQOwK\nOYCeOHGiO/sAAABADxFyAHW73SE/aXp6+iU1AwAAgNgXcgD97ne/G7gPqGVZknTOfUEty5LNZuOe\noAAAALigkAPor371K1VWVmrevHnKzs5WQkKCjh07pvLyck2ZMkXXX399d/YJAACAGBFyAF26dKn+\n9V//VaNHjw4cGzNmjBYvXqz77rtPP/jBD7qjPwAAAMSYkO8D+vHHH6tXr17nHG9tbZXP5+vSpgAA\nABC7Qg6g119/vX7yk59o//79+uijj9TQ0KDdu3drwYIFuummm7qzRwAAAMSQkC/BL1y4UAsWLNDd\nd98tv98v6bP7gE6bNk0PPPBAtzUIAACA2BJyAHW5XPq3f/s3ffzxx/rrX/8qp9Opr33ta0pMTOzO\n/gAAABBjvtB3wX/44Yf67W9/q9/+9rdKSUnR888/r7/85S/d1RsAAABiUMgB9Pjx47rhhhu0e/du\n1dTUqKmpSf/zP/+j73//+zpw4EB39ggAAIAYEnIAXbp0qaZPn66NGzcqISFBkrRkyRJNmzZNy5Yt\n67YGAQAAEFtCDqDHjh3T5MmTzzl+yy236O233+50I6dOnVJJSYlGjRql8ePHa926dZKkxsZGzZ49\nW9nZ2Zo4caI2bdoUeIxlWaqoqNC4ceMC9yRtb28PrNfU1CgvL09ZWVkqKSmRx+MJrB0/flxFRUXK\nyspSYWGhjhw5EljrqCYAAAA6J+QA2qdPH508efKc48eOHVNKSkqnmrAsS/fee68GDx6sP/7xj1qz\nZo2eeOIJHT58WAsXLpTL5dL+/fu1fPlyLVu2LBAWN2zYoN27d2vbtm3asWOHDh8+rLVr10qSTpw4\nobKyMlVWVqq2tlapqamaP3++JKmlpUWlpaWaMmWKDh48qGnTpmnWrFnyer2S1GFNAAAAdE7IAfS2\n227TI488ov/+7/+WJL3xxhvasGGDFi1apFtuuaVTTfzpT3/SBx98oLlz5yohIUFf//rXtXHjRl1+\n+eXatWuX5syZo8TERI0YMUL5+fnaunWrJKm6ulrTp0/XgAED1L9/f5WUlGjLli2SpO3btysvL0+Z\nmZlyOByaO3eu9u7dK4/Ho9raWtntdhUXFyshIUFFRUVKTU3Vnj175PV6O6wJAACAzgn5Nkz33HOP\nevfurV/84hfy+Xy67777lJqaqtLSUk2fPr1TTRw7dkxf//rX9dhjj2n79u1KSkpSaWmp/uEf/kHx\n8fEaNGhQ4Nz09HTt3LlTklRXV6chQ4YErbndblmWpbq6Oo0cOTKwlpycrD59+sjtdsvtdisjIyOo\nh/T0dNXV1enKK6/ssCYAAAA6J+QA+txzz6mgoEC33367mpqa1N7erssuu6xLmmhsbNQf//hHjRs3\nTi+++KJee+01zZw5U6tWrZLD4Qg61+FwqLm5WZLk8/mC1p1Op/x+f+DrQf/+sU6nUz6fT01NTXI6\nned93qampg5rhsJms8n+hW5w9cXZ7bbuLRAmcXGx9brOzilW5xULmFF0YE6RjxlFh0iZU8gB9JFH\nHtHvfvc7felLX5LL5erSJnr16qU+ffqopKREkjRq1CjdcMMNWr58uVpaWoLObW5uDtR3OBxB6z6f\nT/Hx8UpMTDxvaPT5fHK5XHI6neesnX1ep9PZYc1Q9OvXWzYbb8BLkZKSFO4WukXfvr3D3QIughlF\nB+YU+ZhRdAj3nEIOoN/61rf00ksvnXPpuiukp6ervb1d7e3tiouLkyS1t7frqquu0qFDh3Ty5EkN\nHDhQkuR2uwOX3TMyMuR2u5WZmRlYGzx4cNDaWQ0NDWpsbFRGRoa8Xq/Wr18f1IPb7VZ+fr7S0tLU\n1tZ2wZqh+PBDLzugl6ih4Uy4W+hSdrtNffv21unTXvn9VrjbwXkwo+jAnCIfM4oOJufU0aZSyAG0\nV69e+uUvf6lf//rX+upXv3rOZeqNGzdecoPXXnutHA6HnnjiCc2ePVtHjx7VH/7wBz311FP629/+\npoqKCi1evFhvvvmmampqtGrVKknS5MmTtWbNGo0bN07x8fFauXKlCgsLJUn5+fm64447NHXqVA0f\nPlyVlZUaP368kpOTlZOTo9bWVlVVVenWW29VdXW1PB6PcnNz5XK5lJeXd8GaobAsS5+7GxS+gPb2\n2Pyl5fdbMfvaYgUzig7MKfIxo+gQ7jl9oR3Qb33rW93ShMPhUFVVlX72s5/pmmuuUVJSkn76058q\nKytL5eXlKisr04QJE+RyuTRv3rzAjmdxcbE8Ho+KiorU1tamgoICzZgxQ5I0bNgwlZeXa8GCBaqv\nr9fo0aO1dOlSSZ+F6dWrV2vRokWqrKxUWlqaVqxYEbjM3lFNAAAAdI7NsqwLxt+xY8fqueeeC7rP\n54kTJzR48GD16tXLSIPRqL7+k26vERdn0/Qlz3d7HdPWPjwp3C10qbg4m1JSktTQcIYdgQjFjKID\nc4p8zCg6mJxT//4X/rB6h39T8eOPP9bf59Pi4mK9//77XdMZAAAAepwv/FGZDjZMAQAAgIvq5s9q\nAwAAAMEIoAAAADDqop+Cr66uVu/e//9mpX6/XzU1NUEfTJLU6e+DBwAAQM/QYQAdOHDgOTds79ev\nnzZt2hR0zGazEUABAAAQkg4D6AsvvGCqDwAAAPQQ/B1QAAAAGEUABQAAgFEEUAAAABhFAAUAAIBR\nBFAAAAAYRQAFAACAUQRQAAAAGEUABQAAgFEEUAAAABhFAAUAAIBRBFAAAAAYRQAFAACAUQRQAAAA\nGEUABQAAgFEEUAAAABhFAAUAAIBRBFAAAAAYRQAFAACAUQRQAAAAGEUABQAAgFEEUAAAABhFAAUA\nAIBRBFAAAAAYRQAFAACAUQRQAAAAGEUABQAAgFEEUAAAABhFAAUAAIBRBFAAAAAYRQAFAACAUQRQ\nAAAAGEX0tIwYAAAS/0lEQVQABQAAgFEEUAAAABhFAAUAAIBRBFAAAAAYRQAFAACAUQRQAAAAGEUA\nBQAAgFEEUAAAABhFAAUAAIBRERdAPR6PcnJy9OKLL0qSGhsbNXv2bGVnZ2vixInatGlT4FzLslRR\nUaFx48ZpzJgxWrx4sdrb2wPrNTU1ysvLU1ZWlkpKSuTxeAJrx48fV1FRkbKyslRYWKgjR44E1jqq\nCQAAgM6JuAC6YMECnT59OvDzwoUL5XK5tH//fi1fvlzLli0LhMUNGzZo9+7d2rZtm3bs2KHDhw9r\n7dq1kqQTJ06orKxMlZWVqq2tVWpqqubPny9JamlpUWlpqaZMmaKDBw9q2rRpmjVrlrxe70VrAgAA\noHMiKoD+7ne/k9Pp1Je//GVJktfr1a5duzRnzhwlJiZqxIgRys/P19atWyVJ1dXVmj59ugYMGKD+\n/furpKREW7ZskSRt375deXl5yszMlMPh0Ny5c7V37155PB7V1tbKbreruLhYCQkJKioqUmpqqvbs\n2XPRmgAAAOiciAmgbrdbTz31lBYtWhQ49s477yg+Pl6DBg0KHEtPT1ddXZ0kqa6uTkOGDAlac7vd\nsizrnLXk5GT16dNHbrdbbrdbGRkZQfXPPu/FagIAAKBz4sPdgCR9+umn+vGPf6wFCxaob9++geNN\nTU1yOBxB5zocDjU3N0uSfD5f0LrT6ZTf71dra+s5a2fXfT6fmpqa5HQ6z/u8F6sZCpvNJns3R3u7\n3da9BcIkLi62XtfZOcXqvGIBM4oOzCnyMaPoEClziogA+h//8R8aNmyYJkyYEHTc6XSqpaUl6Fhz\nc7NcLpekz4Lh59d9Pp/i4+OVmJh43tDo8/nkcrnkdDrPWTv7vBerGYp+/XrLZuMNeClSUpLC3UK3\n6Nu3d7hbwEUwo+jAnCIfM4oO4Z5TRATQHTt2qL6+Xjt27JAknTlzRj/60Y80c+ZMtbW16eTJkxo4\ncKCkzy7Vn720npGRIbfbrczMzMDa4MGDg9bOamhoUGNjozIyMuT1erV+/fqgHtxut/Lz85WWltZh\nzVB8+KGXHdBL1NBwJtwtdCm73aa+fXvr9Gmv/H4r3O3gPJhRdGBOkY8ZRQeTc+poUykiAuhzzz0X\n9POkSZO0cOFCXXfddTpx4oQqKiq0ePFivfnmm6qpqdGqVaskSZMnT9aaNWs0btw4xcfHa+XKlSos\nLJQk5efn64477tDUqVM1fPhwVVZWavz48UpOTlZOTo5aW1tVVVWlW2+9VdXV1fJ4PMrNzZXL5VJe\nXt4Fa4bCsix97m5Q+ALa22Pzl5bfb8Xsa4sVzCg6MKfIx4yiQ7jnFDEfQrqQ8vJyffrpp5owYYLm\nzJmjefPmBXY8i4uLNWnSJBUVFenGG2/UqFGjNGPGDEnSsGHDVF5ergULFignJ0cffPCBli5dKknq\n1auXVq9erWeeeUZjx47V+vXrtWLFisBl9o5qAgAAoHNslmXxnyldrL7+k26vERdn0/Qlz3d7HdPW\nPjwp3C10qbg4m1JSktTQcIYdgQjFjKIDc4p8zCg6mJxT//6XXXAt4ndAAQAAEFsIoAAAADCKAAoA\nAACjCKAAAAAwigAKAAAAowigAAAAMIoACgAAAKMIoAAAADCKAAoAAACjCKAAAAAwigAKAAAAowig\nAAAAMIoACgAAAKMIoAAAADCKAAoAAACjCKAAAAAwigAKAAAAowigAAAAMIoACgAAAKMIoAAAADCK\nAAoAAACjCKAAAAAwigAKAAAAowigAAAAMIoACgAAAKMIoAAAADCKAAoAAACjCKAAAAAwigAKAAAA\nowigAAAAMIoACgAAAKMIoAAAADAqPtwNAJ931y9eCHcLXW57RWG4WwAAIKKwAwoAAACjCKAAAAAw\nigAKAAAAowigAAAAMIoACgAAAKMIoAAAADCKAAoAAACjCKAAAAAwigAKAAAAowigAAAAMIoACgAA\nAKMIoAAAADCKAAoAAACjCKAAAAAwKmIC6KFDh/T9739f2dnZ+sd//Edt3LhRktTY2KjZs2crOztb\nEydO1KZNmwKPsSxLFRUVGjdunMaMGaPFixervb09sF5TU6O8vDxlZWWppKREHo8nsHb8+HEVFRUp\nKytLhYWFOnLkSGCto5oAAADonIgIoI2Njbr33nt155136uDBg3r88cdVWVmp/fv3a+HChXK5XNq/\nf7+WL1+uZcuWBcLihg0btHv3bm3btk07duzQ4cOHtXbtWknSiRMnVFZWpsrKStXW1io1NVXz58+X\nJLW0tKi0tFRTpkzRwYMHNW3aNM2aNUter1eSOqwJAACAzomIAHry5ElNmDBBBQUFstvt+uY3v6mr\nr75ahw8f1q5duzRnzhwlJiZqxIgRys/P19atWyVJ1dXVmj59ugYMGKD+/furpKREW7ZskSRt375d\neXl5yszMlMPh0Ny5c7V37155PB7V1tbKbreruLhYCQkJKioqUmpqqvbs2SOv19thTQAAAHRORATQ\nYcOG6bHHHgv83NjYqEOHDkmS4uPjNWjQoMBaenq66urqJEl1dXUaMmRI0Jrb7ZZlWeesJScnq0+f\nPnK73XK73crIyAjq4ezzvvPOOx3WBAAAQOfEh7uBv/fJJ5+otLQ0sAu6bt26oHWHw6Hm5mZJks/n\nk8PhCKw5nU75/X61traes3Z23efzqampSU6n87zP29TUdM7jPl8zFDabTfZujvZ2u617C6BLMa/I\ndXY2zCiyMafIx4yiQ6TMKaIC6HvvvafS0lINGjRI//7v/663335bLS0tQec0NzfL5XJJ+iwYfn7d\n5/MpPj5eiYmJ5w2NPp9PLpdLTqfznLWzz+t0OjusGYp+/XrLZuMNiP+vb9/e4W4BF8GMogNzinzM\nKDqEe04RE0CPHTummTNnavLkyXrooYdkt9uVlpamtrY2nTx5UgMHDpQkud3uwKX1jIwMud1uZWZm\nBtYGDx4ctHZWQ0ODGhsblZGRIa/Xq/Xr1wfVd7vdys/Pv2jNUHz4oZcdUAQ5fdorv98Kdxs4D7vd\npr59ezOjCMecIh8zig4m55SSknTBtYgIoB6PRzNnztSMGTN0zz33BI4nJSUpLy9PFRUVWrx4sd58\n803V1NRo1apVkqTJkydrzZo1GjdunOLj47Vy5UoVFhZKkvLz83XHHXdo6tSpGj58uCorKzV+/Hgl\nJycrJydHra2tqqqq0q233qrq6mp5PB7l5ubK5XJ1WDMUlmXpc3eDAuT3W2pv5xdyJGNG0YE5RT5m\nFB3CPaeICKCbN29WQ0ODVqxYoRUrVgSO33nnnSovL1dZWZkmTJggl8ulefPmBXY8i4uL5fF4VFRU\npLa2NhUUFGjGjBmSPvtgU3l5uRYsWKD6+nqNHj1aS5culST16tVLq1ev1qJFi1RZWam0tDStWLEi\ncJm9o5oAAADoHJtlWfxnSherr/+k22vExdk0fcnz3V4Hnbe9olANDWfYEYhQcXE2paQkMaMIx5wi\nHzOKDibn1L//ZRdci4jbMAEAAKDnIIACAADAKAIoAAAAjCKAAgAAwKiI+BQ8EMsKHqwOdwtdbu3D\nk8LdAgAgirEDCgAAAKMIoAAAADCKAAoAAACjCKAAAAAwigAKAAAAowigAAAAMIoACgAAAKMIoAAA\nADCKAAoAAACjCKAAAAAwigAKAAAAo/gueABf2F2/eCHcLXSp7RWF4W4BAHoUdkABAABgFAEUAAAA\nRhFAAQAAYBQBFAAAAEYRQAEAAGAUARQAAABGcRsmAD1ewYPV4W6hy619eFK4WwCAC2IHFAAAAEYR\nQAEAAGAUl+ABAAAuINa++U2KjG9/I4ACQAziD00AkYxL8AAAADCKAAoAAACjuAQPAIgK3C4LiB3s\ngAIAAMAodkABAAiTWPuwGB8UQ6gIoAAAoEvE4l+TQPfgEjwAAACMIoACAADAKAIoAAAAjCKAAgAA\nwCgCKAAAAIwigAIAAMAoAigAAACMIoACAADAKAIoAAAAjCKAAgAAwCgCKAAAAIwigAIAAMAoAigA\nAACMIoBewPHjx1VUVKSsrCwVFhbqyJEj4W4JAAAgJhBAz6OlpUWlpaWaMmWKDh48qGnTpmnWrFny\ner3hbg0AACDqEUDPo7a2Vna7XcXFxUpISFBRUZFSU1O1Z8+ecLcGAAAQ9Qig5+F2u5WRkRF0LD09\nXXV1dWHqCAAAIHbEh7uBSNTU1CSn0xl0zOFwqLm5OaTH22w22bs52tvttu4tAAAAYla4cwQB9Dyc\nTuc5YbO5uVkulyukx6emJnVHW+fYXlFopA4AAIgtffv2Dmt9LsGfx+DBg+V2u4OOud1uDRkyJEwd\nAQAAxA4C6Hnk5OSotbVVVVVVamtr0+bNm+XxeJSbmxvu1gAAAKKezbIsK9xNRKITJ05o0aJFeuON\nN5SWlqZFixYpKysr3G0BAABEPQIoAAAAjOISPAAAAIwigAIAAMAoAigAAACMIoACAADAKAJohDt+\n/LiKioqUlZWlwsJCHTly5Lzn1dTUKC8vT1lZWSopKZHH4zHcac8V6ox+//vf6zvf+Y5GjRqlqVOn\n6tChQ4Y77dlCndNZBw4c0NChQ+X1eg11iFBndOjQId18880aOXKkCgoKdODAAcOd9myhzmnTpk3K\ny8tTdna2br31Vr322muGO8XRo0c7vIVkWLODhYjV3Nxsffvb37Y2bNhgtba2Wps2bbLGjRtnnTlz\nJui8119/3Ro1apR15MgRy+fzWT/5yU+smTNnhqnrniXUGR04cMC6+uqrrePHj1vt7e3Wf/3Xf1nZ\n2dlWQ0NDmDrvWUKd01mnT5+2Jk6caH3jG9+44DnoWqHO6NSpU9bo0aOt5557zvL7/db27dut7Oxs\ny+fzhanznuWL/Lk0duxYq66uzmpvb7dWrlxpTZo0KUxd9zx+v9/atGmTlZ2dbY0dO/a854Q7O7AD\nGsFqa2tlt9tVXFyshIQEFRUVKTU1VXv27Ak6b/v27crLy1NmZqYcDofmzp2rvXv3sgtqQKgzOnXq\nlP75n/9Zw4YNk91u180336y4uDi99dZbYeq8Zwl1TmctWrRI3/ve9wx32bOFOqPq6mpdc801uuGG\nG2Sz2ZSfn6/f/OY3stv548yEUOf0zjvvyO/3q729XZZlyW63y+FwhKnrnufJJ5/UunXrVFpaesFz\nwp0deMdGMLfbrYyMjKBj6enpqqurCzpWV1cX9DWhycnJ6tOnzzlfJ4quF+qMbrrpJt19992Bn195\n5RV5vd5zHovuEeqcJGnbtm36+OOPddttt5lqDwp9RseOHdPll1+u2bNn6+qrr9Ytt9yi9vZ29erV\ny2S7PVaoc8rNzdWVV16pG2+8UcOHD9fKlSu1bNkyk632aFOnTlV1dbWGDx9+wXPCnR0IoBGsqalJ\nTqcz6JjD4VBzc3PQMZ/Pd85/WTqdTvl8vm7vsacLdUaf99Zbb2nOnDmaM2eOUlJSurtFKPQ5nTx5\nUo8//rh+/vOfm2wPCn1GjY2N2rRpk2677Tbt27dPkydP1j333KPGxkaT7fZYoc6ppaVFQ4YM0ebN\nm/Xqq69q+vTpuu+++zr83YiuM2DAANlstg7PCXd2IIBGMKfTec6btbm5WS6XK+jYhULp35+Hrhfq\njM7at2+fbrvtNt1+++265557TLQIhTYnv9+vhx56SD/84Q91+eWXm26xxwv1vdSrVy+NHz9eubm5\nSkhI0O233y6Xy6XDhw+bbLfHCnVOTzzxhK644goNHz5ciYmJmj17ttra2rR//36T7aID4c4OBNAI\nNnjw4HO2wt1ud9CWuSRlZGQEndfQ0KDGxkYu7xoQ6owk6emnn9acOXNUVlame++911SLUGhzOnXq\nlP70pz9p0aJFGj16tCZPnixJmjBhAncsMCDU91J6erpaW1uDjvn9fll8q7QRoc7p5MmTQXOy2WyK\ni4tTXFyckT5xceHODgTQCJaTk6PW1lZVVVWpra1NmzdvlsfjOeeWCvn5+dq5c6cOHTqklpYWVVZW\navz48UpOTg5T5z1HqDM6cOCAHn30Ua1atUr5+flh6rbnCmVOAwcO1NGjR3Xo0CEdOnRI27ZtkyTt\n2bNHo0ePDlfrPUao76XCwkLt27dPu3fvlt/vV1VVlVpaWnT11VeHqfOeJdQ5TZw4UZs3b9axY8f0\n6aef6qmnnlJ7e7uys7PD1Dn+Xtizg7HP2+OSvP7669Ytt9xiZWVlWYWFhdarr75qWZZlLVy40Fq4\ncGHgvGeeecb6zne+Y40cOdK6++67LY/HE66We5xQZjRjxgxr6NChVlZWVtA/e/bsCWfrPUqo76Wz\n3nvvPW7DZFioM9q7d69VWFhoZWVlWTfffLN15MiRcLXcI4UyJ7/fb61cudK67rrrrOzsbOuOO+6w\n3njjjXC23SPV1tYG3YYpkrKDzbK4bgEAAABzuAQPAAAAowigAAAAMIoACgAAAKMIoAAAADCKAAoA\nAACjCKAAAAAwigAKAAAAowigAAAAMIoACgAAAKP+H50rxJpMPhIiAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f6fa167d828>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pd.DataFrame(features_test).plot.hist()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
